package net.bwie.realtime.jtp.dwd.log.job;

import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.dwd.log.function.AdjLogIsNewProcessFuntion;
import net.bwie.realtime.jtp.dwd.log.function.AppLogSplitProcessFunction;
import net.bwie.realtime.jtp.utils.KafkaUtil;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class JtpLogEtlJob {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataStream<String> kafkaStream = KafkaUtil.consumerKafka(env, "topic-log");

        DataStream<String> pageStream = processLog(kafkaStream);
        pageStream.print();

        KafkaUtil.producerKafka(pageStream, "dwd-traffic-page-log");

        env.execute("JtpLogEtlJob");
    }

    private static DataStream<String> processLog(DataStream<String> kafkaStream) {
        DataStream<String> cleanedStream = appLogCleaned(kafkaStream);
        DataStream<String> isNewStream = isNewStream(cleanedStream);
        DataStream<String> splitStream = appLogSplit(isNewStream);
        return splitStream;
    }

    private static DataStream<String> appLogSplit(DataStream<String> isNewStream) {
        final OutputTag<String> errTag = new OutputTag<String>("errTag") {};
        final OutputTag<String> startTag = new OutputTag<String>("startTag") {};
        final OutputTag<String> displayTag = new OutputTag<String>("displayTag") {};
        final OutputTag<String> actionTag = new OutputTag<String>("actionTag") {};

        SingleOutputStreamOperator<String> process =isNewStream.process(new AppLogSplitProcessFunction(errTag, startTag, displayTag, actionTag));

        SideOutputDataStream<String> startStream = process.getSideOutput(startTag);
        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-log");
        SideOutputDataStream<String> displayStream = process.getSideOutput(displayTag);
        KafkaUtil.producerKafka(displayStream,"dwd-traffic-display-log");
        SideOutputDataStream<String> actionStream = process.getSideOutput(actionTag);
        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-log");
        SideOutputDataStream<String> errorStream = process.getSideOutput(errTag);
        KafkaUtil.producerKafka(errorStream,"dwd-traffic-error-log");
        return process;
    }

    private static DataStream<String> isNewStream(DataStream<String> cleanedStream) {
        KeyedStream<String, String> midStream = cleanedStream.keyBy(new KeySelector<String, String>() {
            @Override
            public String getKey(String s) throws Exception {
                return JSON.parseObject(s).getJSONObject("common").getString("mid");
            }
        });

        SingleOutputStreamOperator<String> isNewStream = midStream.process(new AdjLogIsNewProcessFuntion());

        return isNewStream;
    }

    private static DataStream<String> appLogCleaned(DataStream<String> kafkaStream) {
        OutputTag<String> dirtyTag = new OutputTag<String>("dirty") {};
        SingleOutputStreamOperator<String> cleanedStream = kafkaStream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String s, ProcessFunction<String, String>.Context context, Collector<String> collector) throws Exception {
                try {
                    JSON.parseObject(s);
                    collector.collect(s);
                } catch (Exception e) {
                    context.output(dirtyTag, s);
                }
            }
        });
        return cleanedStream;
    }
}
